#!/usr/bin/python
# -*- coding: latin-1 -*-

import sys
import os
import threading
import mimetypes
import smtplib
import errno
import shutil
import re
import time

import Queue
from EMLsqlite import Database


# ----version sqlite------------------------------------------------------------------------------------------------------------------------------------

#guardar en base de datos sqlite
def rellenarBD(filepath,keyword):
	#para cada "linea" del grep
	#	guardar ruta fichero en files
	#	obtener el id
	#	guardar id + keyword en tags
	global databasename

	db = Database(databasename)
	try:
		lastidfile = db.get_id_file(filepath)
		if (lastidfile == -1):
			lastidfile = db.add_file(filepath)
		
		lastidkeyword = db.get_id_keyword(keyword)
		if (lastidkeyword == -1):
			lastidkeyword = db.add_keyword(keyword,'empty regexp')
			#print "	                ++++++added keyword: %s with id %d" % (keyword,lastidkeyword)
		
		db.add_tag(lastidkeyword,lastidfile)
		#print "	                ++++++added tag: %s and file %s with id's %d %d" % (keyword, filepath, lastidkeyword, lastidfile)

		
		
		db.close()
	except Exception, e:
		db.close()
		print "[rellenarDB ] error:%s con  %s y keyword:%s" % (e,filepath,keyword);
		


	#problema con keyword, se guarda el nombre del keyword? o la grep regexp?
	#crear una tabla tags con nombre keyword y regexp? y otra tabla con fileid,tagid?

def leerListadoSqlite(fich,output):
    """ 
	Lee un fichero con resultados de busquedas, para cada ruta de fichero
	inserta en la bd


    #fich = ruta completa del fichero
    #output = solo el nombre del fichero. Sera utilizado en rellenarBD como keyword en la base de datos (quitandole el .txt)
    """

    try:
        #dirdestino = os.path.abspath(output)  
        #if (not os.path.isdir(dirdestino)):
        #    mkdirRecursivo(dirdestino)

        print "[leerListado] begin"

        f = open(fich,'r')
        
        for line in f.read().split('\n'):
            #si empeiza por Binary file -> filtrar eso
            if (not line.startswith('/')):
                line = line.split('/')
                line = '/' + line[1]

            line2=line
            nombreIni=line2
	    
	    n2 = output.split('.txt')
            output =  n2[0]
           
            try:

                #print "[leerListado] rellenar db nombre %s output %s" % (UnicodeOrIso8859(nombreIni), output )
                rellenarBD(UnicodeOrIso8859(nombreIni), output )
                #print "[leerListado] rellenar db end"
                
            except Exception, e:
                print "[leerlistado sqlite] error1-  file:%s %s" % (fich,e);
            
        f.close()
        print "[leerListado] end"
      
    except Exception, e:
                   print "[leerListado] ERROR2 %s TRANTANDO FICHERO %s" % (e,fich)
                   


def alphabeticalOrder(filelist):
	""" ordenar alphabeticamente la lista de rutas de ficheros """
	return sorted(filelist)
	


def reportWriteToDisk(keywordlist,combination,filelist,dirdestino,copy=False):
  
        #unicode problems------------
        #UnicodeOrIso8859(keyword) 
        #query.encode("UTF-8")

	reportname = ''
	reporttitle = ''
	index = 0
	for kw in keywordlist:
		if (index == 0):
		
			if (combination == 'NOT'):
				reportname = 'NOT(' + kw
				reporttitle = 'NOT(' + kw
			else:
				reportname = kw
				reporttitle = kw
		else:
			if (combination == 'NOT'):
				comb = 'AND'
			else:
				comb = combination
			
			reporttitle = reportname + ' ' + comb + ' ' + kw
			reportname = reportname + '_' + comb + '_' + kw
		index = index + 1
	
	if (combination == 'NOT'):
		reportname = reportname + ')'
		reporttitle = reporttitle + ')'
		

	print "keywordlist elems: %d filelist elems: %d " % (len(keywordlist),len(filelist))

	print "[reportWriteToDisk] reportname: "+reportname
	
	try:
          #print "0"
          FILE = open("%s/report_%s.html" % (dirdestino,reportname) ,"w")
          #print "1"
          
          
          FILE.write("<html><head><title>%s</title></head><body><h3>%s</h3><table><tbody>" % (reporttitle,reporttitle))	

          print "antes de for keywordlist elems: %d filelist elems: %d " % (len(keywordlist),len(filelist))
          for filepath in filelist:
                  
                  #print ("<tr><td><a href='%s'>%s</a></td></tr>" % (filepath,filepath))
                  #isascii = is_ascii(filepath)
                  #if (isascii):
                  #   print "filepath %s is ascii" % filepath
                  #else:
                  #   line = "filepath %s is not ascii " % unicode(filepath,'iso-8859-1')
                  #   #print line.encode("UTF-8")
                  #   print line.encode("iso-8859-1")

                  

                  
                  if (copy):
                    #calcular path destino (dirdestino + '/' + 'resultados' + '/' + <ruta extraida del correo>)
                    #----extraer ruta relativa a pst (la ruta debe contener el directorio eml , sino no funciona
                    
                    #print "1"
                    rutaOriginal = filepath
                    rutaParte2 = rutaOriginal
                    
                    if (len(rutaOriginal.split('eml/')) > 1):
                      rutaOriginal = rutaOriginal.split('eml/')
                      rutaParte2 = '/' + rutaOriginal[1]
                      #----calcular ruta destino
                      
                      #----mirar si fichero existe
                      #----si no existe --> copiar
                      
                    rutaNueva = dirdestino + '/' + 'reports' + rutaParte2
                    
                    #print "2"
                    
                    if (not os.path.exists(rutaNueva)):
                      #quitar nombre del fichero al hacer mkdirRecursivo!!!
                      mkdirRecursivo(os.path.dirname(rutaNueva))
                      shutil.copy2(filepath,rutaNueva )
                    #report en "dirdestino" con ruta relativa a resultados -> esto se podria modificar en la bd?
                    rutaRelativa = './' + 'reports' + rutaParte2
                    
                    print "3"
                    
                    line = rutaRelativa.encode("iso-8859-1")
                  
                  else:
                    line = filepath.encode("iso-8859-1")
                    
                  print " dentro de for keywordlist elems: %d filelist elems: %d " % (len(keywordlist),len(filelist))
                  
                  line = "<tr><td><a href='%s'>%s</a></td></tr>" % (line,line)
                  print line
                  FILE.write(line)
                  
                  

          #print "3"
          FILE.write("</tbody></table></body><html>")
          FILE.close()
          
        except Exception, e:
          print "[reportWriteToDisk] exception %s " % e
          raise
          

def leerDirectorioSqlite(target,output):
	
    """
	Lee directorio con ficheros de resultados de busqueda.
	cada fichero contiene listas de ficheros eml (con su ruta)
	asociados a keywords.
    """
    global databasename

    print "[leerDirectorioSqlite] entrada"
    try:
      db = Database(databasename)
      db.initDatabase()

          
      #listar ficheros de un directorio
      dirList=os.listdir(target)

      for fname in dirList:
          print "[leerDirectorioSqlite] file %s" % fname
          nombre = os.path.abspath(target) + '/' + fname
          leerListadoSqlite(nombre,fname)


    
      #db = Database(databasename)
      #kwl = db.keyword_list()

      #for keyword in kwl:
          #print "-----resultado keyword: %s " % keyword[0]
          #tgl = db.tag_list(keyword[0])
          #report(keyword[0],tgl)




      print "[leerDirectorioSqlite] end"
      db.close()
    except Exception, e:
      print "[leerDirectorioSqlite] target %s , output %s, error %s " % (target,output,e)




















#-------------------------------------copiado-----------------------------------------------------------
def mkdirRecursivo(directorio):
	""" creacion de directorios y subdirectorios """
        aux = []
        while (not os.path.exists(directorio)):
                #print "el directorio %s no existe" % directorio
                aux.append(directorio)
                directorio = os.path.dirname(directorio)
                #print "directorios a crear: %d " % len(aux)

        
        aux.reverse()
        
        for eldir in aux:
                try:
                        os.mkdir(eldir)
                except Exception, e:
                        print "[mkdirRecursivo] error creando directorio %s error %s " % (eldir,e)



        
                    

def leerListado(fich,output):

    try:
        dirdestino = os.path.abspath(output)
        
        
        if (not os.path.isdir(dirdestino)):
            mkdirRecursivo(dirdestino)
            

        
        f = open(fich,'r')
        
        for line in f.read().split('\n'):
            
            

            #si empeiza por Binary file -> filtrar eso
            if (not line.startswith('/')):
                line = line.split('/')
                line = '/' + line[1]
            
            

            line2=line
            nombreIni=line2
            #if (not line.endswith('.eml')):
                #final de linea con el nombre del fichero (aunque ahora solo tendria que estar el nombre del ficehro y nada mas
                #line2 = line.split('.eml')
                #nombreIni = line2[0] + '.eml'
                #print(nombreIni)

            #print "control point 0"
            #recrear el path desde ./eml/ (quitando /eml)
            nombreRecreado = nombreIni
            dirfinal = dirdestino +  UnicodeOrIso8859(os.path.dirname(nombreRecreado))
            #print "control point 1"

            if (not (nombreIni.find('/eml/') == -1)):
                nombreRecreado = nombreIni.split('/eml/')
                dirfinal = dirdestino + '/' + UnicodeOrIso8859(os.path.dirname(nombreRecreado[1]))
                #print "directorio FinAL:  %s  \n\n" % dirfinal
                
            #print "control point 2"
            
            
            if (not (nombreIni.find('/EMLextractor/') == -1)):
                nombreRecreado = nombreIni.split('/EMLextractor/')
                dirfinal = dirdestino + '/' + UnicodeOrIso8859(os.path.dirname(nombreRecreado[1]))
                #print "directorio FinAL:  %s  \n\n" % dirfinal


            #print "control point 3"

            
            
            #comprobaciones
            if (not os.path.isdir(dirfinal)):
                mkdirRecursivo(dirfinal)

           
            try:
                #print "[leerListado]copiando: %s  %s " % (UnicodeOrIso8859(nombreIni), dirfinal )           
                #print "control point 4"
                shutil.copy2(UnicodeOrIso8859(nombreIni), dirfinal )
                
            except Exception, e:
                print "[leerListado] error1-  file:%s %s" % (fich,e);
            
        f.close()
      
    except Exception, e:
                   print "[leerListado] ERROR2 %s TRANTANDO FICHERO %s" % (e,fich)
                    
    
def leerDirectorio(target,output):
    
    dirdestino = os.path.abspath(output)
    #print "output1: %s " % dirdestino  
        
    if (not os.path.isdir(dirdestino)):
        mkdirRecursivo(dirdestino)
        
    #listar ficheros de un directorio
    dirList=os.listdir(target)
    for fname in dirList:
        #print fname
        nombre = os.path.abspath(target) + '/' + fname
        n2 = fname.split('.txt')
        directoriofinal = os.path.abspath(dirdestino) + '/' + n2[0]
        #print nombre
        #print directoriofinal

        #llmar a leerListado(nombre, dirdestino + '/' +nombresintxt);
        leerListado(nombre,directoriofinal)
        #print "control point 5. copiando el listado de resultados"
        try:
            #print "[leerDirectorio]copiando: %s  %s " % (nombre, directoriofinal )  
            shutil.copy2(nombre, directoriofinal )
        except Exception, e:
            print "[leerDirectorio copiar listado resultados] - dir:%s %s" % (fname,e);
        
    



def is_ascii(s):
    try:
        s.decode('ascii')
    except UnicodeDecodeError:
        print "it was not a ascii-encoded unicode string"
        return False
    else:
        print "It may have been an ascii-encoded unicode string"
        return True

def UnicodeOrIso8859(encodedstr):
  try:
    result = encodedstr.decode("UTF-8")
  except UnicodeDecodeError:
    result = encodedstr.decode("ISO-8859-1")
  return result
  
def encodeIso8859OrUTF8(encodedstr):
  try:
    result = encodedstr.encode("ISO-8859-1")
  except UnicodeDecodeError:
    result = encodedstr.encode("UTF-8")
  return result
#-----------------------------------busquedas--------------------------------------------------------

   
class GrepSearchThread(threading.Thread):
    def __init__(self, queue):
        """
	busqueda mediante comando grep en una clase thread. Obtiene los parametros de busqueda de queue. No se puede escribir a bd directamente, porque grep solo nos permite escribir a disco... A priori        

        @param queue: queue from where parameters of one search are retrieved
        """
        threading.Thread.__init__(self)
        self.queue = queue
        
        
        

    def run(self):
        """Overrides Thread.run. Don't call this directly its called internally
        when you call Thread.start().
        """
        
        
        arrayinfos = self.queue.get()
        
        #extra
        self.keyword=arrayinfos['keyword']
        self.dirsearch= arrayinfos['dirsearch']
        self.output = arrayinfos['output']
        self.grepparams = arrayinfos['grepparams']
        
        
        try:

          query = "grep "+UnicodeOrIso8859(self.grepparams)+" "+ UnicodeOrIso8859(self.keyword) +" "+UnicodeOrIso8859(self.dirsearch)+" > "+UnicodeOrIso8859(self.output)
          
          os.system(query.encode("UTF-8"))
         
          print "BUSQUEDA  %s  FINALIZADA -------------\n" % (query)
          #signals to queue job is done
          self.queue.task_done()
          
        except Exception, e:
          print "[GrepSeachThread] error %s " % e
          #signals to queue job is done
          self.queue.task_done()
        
        
        

#no se utiliza ver grepSearchThread
def lanzarBusqueda(keyword,directory,output):

    global queue
    
    try:
      
      #print "Bparametros keyword %s directory %s output %s " % (output,UnicodeOrIso8859(keyword),directory)
      #print "grepsearch  = %s " % (grepsearch)

        
      #poner en la queue
      arrayinfos={}
      arrayinfos['keyword']=keyword
      arrayinfos['dirsearch']=directory
      arrayinfos['output']= output
      arrayinfos['grepparams']= ' -ilR '
    
      queue.put(arrayinfos)

    except Exception, e:
      print "[busqueda de %s] error: %s" % (keyword,e)




def leerLista(target,outdir):


    global queue
    queue = Queue.Queue()
    
    
    print "[leerLista queue threads] parametros target %s y outdir %s " % (target, outdir)
    
    print "[leerLista queue threads] preparando 7 threads "
    #spawn a pool of threads, and pass them queue instance 
    for i in range(10):
      t = GrepSearchThread(queue)
      t.setDaemon(True)
      t.start()

    try:
        dirdestino = os.path.abspath(outdir);
        dirdestino = os.path.abspath(dirdestino + '/' + 'outputfindings')
        
        if (not os.path.isdir(dirdestino)):
            mkdirRecursivo(dirdestino)
        
        #cutrada para ir rapido
        indice= 0
        keyword = ''
        directory = ''
        output = ''
        
        f = open(target,'r')
        #print "before for in [leerlista]"
        for line in f.read().split('\n'):
        
            print line
            #extraer los 3 parametros
             
            if ('#'  in line):
                indice= 0
                keyword = ''
                directory = ''
                output = ''
            elif (indice==0):
                keyword = line
                indice+=1
            elif (indice==1):   
                directory = line
                indice+=1
            elif (indice==2):   
                output = line
                indice+=1
                #lanzar grep o similar en python
                #print "keyword %s " % keyword
		if (not output.endswith(".txt")):
			output=output+".txt"
                #rellena la queue i sale
                lanzarBusqueda(keyword,directory,dirdestino + '/' + output)
                indice= 0
                keyword = ''
                directory = ''
                output = ''
            
                
            #print "keyword %s directory %s output %s index %d" % (keyword,directory,output,indice)
            
        #wait on the queue until everything has been processed     
        queue.join()
        print "queu empty!!"
        
        f.close()
        
        
        
        return dirdestino
        
    except Exception, e:
        print " [leerLista] - target: %s error %s" % (target,e)
        
#-----------------------------------------------------module--main-----------------------------------------------------        
def moduleMain (sourcefile, dirdest, typesearch,dbname,keywordlist='',combination='',copyfiles = True):
  global grepsearch
  global databasename


  print ("[ModuleMain] with filesource %s dirdest %s searchtype %s and casename %s " % (sourcefile, dirdest, typesearch,dbname))




  #dbname
  databasename = os.path.abspath(dirdest) + '/' + dbname
  if (not databasename.endswith('.db')):
	databasename = databasename + '.db'

  try:
    db = Database(databasename)
    db.close()
  except Exception, e:
    print "[moduleMain] create database %s " % e
 
    
  try:
    db = Database(databasename)
    db.initDatabase()
    db.close()
  except Exception, e:
    print "[moduleMain] initdatabase %s " % e
 



  #RETRIEVING ONLY THE KEYWORDS
  if (typesearch == "getkeywords"):
	try:
		print "[modulemain - getkeywords]" 
		db = Database(databasename)
		listresult  = db.keyword_list()
		db.close()
		return listresult 
	except Exception,e:
		print "[moduleMain] %s" % (e)
                return 0

  if (typesearch == "createreport" ):
	try:
		print "[moduleMain - createreport]"
		print "[moduleMain - createreport] combination %s ;" % combination
		db = Database(databasename)
		result = []
		try:
			result  = db.report(keywordlist,combination)
			print "report result %d elems" % len(result)
		except Exception,e:
			print "[createreport]__[report] failed: %s" % e
			return "error"


		try:
			result = db.addParentFiles(result)
		except Exception,e:
			print "[createreport]__[addParentFiles] failed: %s" % e

		try:
			result = alphabeticalOrder(result)
		except Exception,e:
			print "[createreport]__[addParentFiles] failed: %s" % e
			
		print "[main]keywordlist elems: %d filelist elems: %d " % (len(keywordlist),len(result))
	    	reportWriteToDisk(keywordlist,combination ,result,dirdest,copyfiles)
		#print (keywordlist)
		db.close()
		return result 
	except Exception,e:
		print "[moduleMain] %s" % (e)
                return 0
		
  #Search done, db work to report
  
  if (typesearch == "dbwork"):
    print ("[moduleMain - post search, db work] starting search from case %s " % databasename)
    grepsearch = True
    try:
      
        dirlistas = os.path.abspath(dirdest);
        dirlistas = os.path.abspath(dirlistas + '/' + 'outputfindings')
        # bd para crear reports
        leerDirectorioSqlite(dirlistas,dirdest)
        return 1

    except Exception, e:
        print "[moduleMain] %s" % (e)
        return 0

  #NORMAL SEARCH
  
  if (typesearch == "grep"):
    print ("[moduleMain - grep search] starting search from case %s " % databasename)
    grepsearch = True
    try:
      
        start = time.time()
        
	#leer lista con output en el directorio dirdestino + / + listas
    	dirlistas = leerLista(sourcefile,dirdest)
    	# bd para crear reports
    	leerDirectorioSqlite(dirlistas,dirdest)
    	
    	print "Elapsed Time: %s" % (time.time() - start)
    	
    	return 1

    except Exception, e:
    	print "[moduleMain] %s" % (e)
    	return 0
  else:
    grepsearch =  False
    
 


#-----------------------------------------------------script--main-----------------------------------------------------
if __name__ == "__main__":
      sys.stderr.write('This is a module. It must be called from inside a python script. ')
      sys.exit(1)

